# Copyright (c) 2020 Presto Labs Pte. Ltd.
# Author: jhkim

import collections
import datetime
import glob
import os
import pandas
import shutil
import signal
import subprocess
import sys
import tabulate
import tempfile

from concurrent.futures import ProcessPoolExecutor, as_completed

from absl import app
import logging

from coin.base import flags
from coin.base.mail_util import send_mail
from cc.appcoin2.strategy.lm_sim_backtest import LmSimBacktest
from cc.appcoin2.strategy.dm_sim_backtest import DmSimBacktest
from cc.appcoin2.strategy.dmm_sim_backtest import DmmBacktest
from cc.appcoin2.strategy.rmm_sim_backtest import RmmBacktest
from cc.appcoin2.strategy.umm_sim_backtest import UmmBacktest
from cc.appcoin2.strategy.vmm_sim_backtest import VmmBacktest
from cc.appcoin2.strategy.hamm_sim_backtest import HammBacktest
import cc.appcoin2.strategy.run_sim_util as rsutil
from cc.appcoin2.strategy.sim_driver import chunks
from cc.appcoin2.strategy.sim_driver import cleanup_slurm_jobs

from coin.exchange.kr_rest.product.product_impl import generate_product_from_str2
import coin2.exchange.order.system_pb2 as ospec
import pyorio.record.simple_record as py_impl


def guess_sim_type(config_file):
  config_name = os.path.basename(config_file)
  if config_name.startswith('basis_smm'):
    sim_type = 'basis_smm'
  elif config_name.startswith('basis_spread'):
    sim_type = 'basis_spread'
  elif config_name.startswith('mm_currency'):
    sim_type = 'basis_spread'
  elif config_name.startswith('mm_portfolio'):
    sim_type = 'basis_spread'
  elif config_name.startswith('mmrev'):
    sim_type = 'mmrev'
  elif config_name.startswith('basis_lmagg'):
    sim_type = 'basis_lmagg'
  elif config_name.startswith('basis_model'):
    sim_type = 'basis_lmagg'
  elif config_name.startswith('model'):
    sim_type = 'basis_lmagg'
  elif config_name.startswith('rmm'):
    sim_type = 'rmm'
  elif config_name.startswith('umm'):
    sim_type = 'umm'
  elif config_name.startswith('vmm'):
    sim_type = 'vmm'
  elif config_name.startswith('dmm'):
    sim_type = 'dmm'
  elif config_name.startswith('hamm'):
    sim_type = 'hamm'
  flags.FLAGS.sim_type = sim_type


def get_sim_backtests_for_a_config(
    sim_type,
    config_file,
    sparams,
    sparam_mdl,
    tmp_work_root=None):
  sim_backtests = []
  nickname = config_file.split('/')[-1].replace('.json', '')
  '''
  replace_dict = {'huobi': 'hb', 'okex': 'ok', 'futures': 'fut', 'allexpiry': 'alle'}
  for key, value in replace_dict.items():
    nickname = nickname.replace(key, value)'''

  strat_prefix = flags.FLAGS.strat_prefix or '' + nickname
  sim_backtest_cls = None
  if sim_type == 'lm':
    sim_backtest_cls = LmSimBacktest
  elif sim_type == 'dm':
    sim_backtest_cls = DmSimBacktest
  elif sim_type in ['basis_smm', 'basis_smm2']:
    from cc.appcoin2.strategy.basis_strat_backtest.basis_smm_sim_backtest import BasisSmmBacktest
    sim_backtest_cls = BasisSmmBacktest
  elif sim_type in ['basis_lmagg', 'basis_lmagg2']:
    from cc.appcoin2.strategy.basis_strat_backtest.basis_lmagg_sim_backtest import BasisLmaggBacktest
    sim_backtest_cls = BasisLmaggBacktest
  elif sim_type == 'mmrev':
    from cc.appcoin2.strategy.basis_strat_backtest.mmrev_sim_backtest import MmrevBacktest
    sim_backtest_cls = MmrevBacktest
  elif sim_type in ['basis_spread', 'basis_spread2']:
    from cc.appcoin2.strategy.basis_strat_backtest.basis_spread_sim_backtest import BasisSpreadBacktest
    sim_backtest_cls = BasisSpreadBacktest
  elif sim_type in ['basis_spread_pass']:
    from cc.appcoin2.strategy.basis_strat_backtest.basis_spread_pass_sim_backtest import BasisSpreadPassBacktest
    sim_backtest_cls = BasisSpreadPassBacktest
  elif sim_type in ['rmm']:
    sim_backtest_cls = RmmBacktest
  elif sim_type in ['umm']:
    sim_backtest_cls = UmmBacktest
  elif sim_type in ['vmm']:
    sim_backtest_cls = VmmBacktest
  elif sim_type in ['dmm']:
    sim_backtest_cls = DmmBacktest
  elif sim_type in ['blp']:
    from cc.appcoin2.strategy.basis_strat_backtest.blp_sim_backtest import BLPSimBacktest
    sim_backtest_cls = BLPSimBacktest
  elif sim_type in ['delta']:
    from cc.appcoin2.strategy.basis_strat_backtest.delta_mgr_sim_backtest import DeltaMgrSimBacktest
    sim_backtest_cls = DeltaMgrSimBacktest
  elif sim_type in ['hamm']:
    sim_backtest_cls = HammBacktest
  else:
    raise ValueError(sim_type)

  for sparam in sparams:
    tmp_dirname = tempfile.mkdtemp(dir=tmp_work_root)
    sim_backtest = sim_backtest_cls(tmp_dirname=tmp_dirname,
                                    analyzer_feed_machine=flags.FLAGS.analyzer_feed_machine,
                                    config_path=config_file,
                                    strat_prefix=strat_prefix,
                                    **sparam,
                                    **sparam_mdl,
                                    pick_symbol_regex=flags.FLAGS.pick_symbol_regex,
                                    unfold_stat=len(sparams) == 1,
                                    unfold=flags.FLAGS.unfold,
                                    use_live_latency=flags.FLAGS.use_live_latency,
                                    use_live_price=flags.FLAGS.use_live_price,
                                    use_adjusted_pos=flags.FLAGS.use_adjusted_pos,
                                    use_latency_model=flags.FLAGS.use_latency_model,
                                    exclude_trading_halt=flags.FLAGS.exclude_trading_halt,
                                    bbo_from_cache=flags.FLAGS.bbo_from_cache,
                                    include_funding_rate=flags.FLAGS.include_funding_rate,
                                    run_with_kline_dumper=flags.FLAGS.run_with_kline_dumper,
                                    warmup_hours=flags.FLAGS.warmup_hours,
                                    training_date=flags.FLAGS.training_date,
                                    force_plot_debug_info=flags.FLAGS.force_plot_debug_info,
                                    include_mm_stats=flags.FLAGS.include_mm_stats,
                                    include_lp_plot=flags.FLAGS.include_lp_plot,
                                    price_bp_tholds=flags.FLAGS.price_bp_tholds,
                                    quote_tholds=flags.FLAGS.quote_tholds,
                                    base_tholds=flags.FLAGS.base_tholds,
                                    include_option_pnl=flags.FLAGS.include_option_pnl,
                                    include_tag_info=flags.FLAGS.include_tag_info,
                                    use_slimfeed=flags.FLAGS.use_slimfeed)
    sim_backtest.sparam = sparam  # just for later groupby use
    sim_backtests.append(sim_backtest)
  return sim_backtests


def get_jobnames(sim_backtests):
  jobnames = []
  for sim_backtest in sim_backtests:
    jobnames.extend(sim_backtest.gen_jobnames())
  return jobnames


PRERUN = 1
KLINE = 3
RUN = 2


def get_cmds(sim_backtests, runtype, skip_last=False):
  cmds = []
  kline_prefix_check = set()
  for sim_backtest in sim_backtests:
    if runtype == PRERUN:
      cmds.extend(sim_backtest.gen_cmds_prerun())
    elif runtype == KLINE:
      if hasattr(sim_backtest, "kline_prefix") and sim_backtest.kline_prefix not in kline_prefix_check:
        kline_prefix_check.add(sim_backtest.kline_prefix)
        cmds.extend(sim_backtest.gen_cmds_kline())
    else:
      cmds.extend(sim_backtest.gen_cmds())
    if skip_last:
      cmds = cmds[:-1]
  return cmds


def parse_proto_data(data):
  record = ospec.OrderRecordProto()
  record.ParseFromString(data)
  return record


def read_one_pb(pb_path):
  record_reader = py_impl.SimpleRecordReader(pb_path)
  first_acc_info = None
  last_data = None
  while data := record_reader.read_record():
    if first_acc_info is None:
      first_acc_info = parse_proto_data(data).account_info
    last_data = data
  if last_data is None:
    return None
  last_acc_info = parse_proto_data(last_data).account_info
  market = first_acc_info.market_type
  exchange = first_acc_info.exchange
  api = first_acc_info.api
  dt = datetime.datetime.fromtimestamp(first_acc_info.start_timestamp / 1e9)
  product = generate_product_from_str2(market, exchange, api, first_acc_info.relative_norm, dt)
  relative_norm = product.subscription_symbol.replace('CURRENT_', '')
  abs_symbol = product.symbol
  mea = f'{market}.{exchange}.{api}'
  record_reader.close()
  end_pos = 0 if abs(last_acc_info.end_position) < 1e-6 else last_acc_info.end_position
  return relative_norm, abs_symbol, end_pos, dt, mea


def read_ending_pos(sim_backtest):
  paths = os.path.join(sim_backtest.tmp_dirname, sim_backtest.pb_suffix)
  pb_paths = list(sorted(glob.glob(paths)))
  print("path: ", paths)
  init_pos = collections.defaultdict(lambda: collections.defaultdict(list))
  abs_norms = collections.defaultdict(lambda: collections.defaultdict(list))
  futures = []
  with ProcessPoolExecutor(max_workers=6) as executor:
    for pb_path in iter(pb_paths):
      futures.append(executor.submit(read_one_pb, pb_path))
    data = []
    for fut in as_completed(futures):
      result = fut.result()
      if result is None:
        continue
      data.append(result)
    interval_count = sim_backtest.trading_interval_count()
    for res in data:
      relative_norm, abs_symbol, end_pos, init_ts, mea = res
      if relative_norm not in init_pos[mea]:
        init_pos[mea][relative_norm] = [0 for _ in range(interval_count)]
        abs_norms[mea][relative_norm] = [None for _ in range(interval_count)]
      interval_idx = sim_backtest.trading_interval_map[init_ts]
      assert init_pos[mea][relative_norm][interval_idx] == 0
      init_pos[mea][relative_norm][interval_idx] = end_pos
      abs_norms[mea][relative_norm][interval_idx] = abs_symbol

  for mea, mea_init_pos in init_pos.items():
    for k, v in mea_init_pos.items():
      mea_init_pos[k] = [0] + v[:-1]
    for real_norm, pos in mea_init_pos.items():
      abs_norm = abs_norms[mea][real_norm]
      assert len(pos) == len(abs_norm)
      for i in range(len(pos)):
        if i > 0 and i < len(abs_norm) - 1 and abs_norm[i] != abs_norm[i - 1]:
          pos[i] = 0
  init_pos = dict(init_pos)
  print("init_position", init_pos)
  sim_backtest.set_adjusted_init_pos(init_pos)
  return True


def run_sim_backtests(sim_backtests, batch_runner, num_cpu):
  jobnames = get_jobnames(sim_backtests)
  use_adjusted_pos = flags.FLAGS.use_adjusted_pos \
      and sim_backtests[0].trading_interval_count() > 1
  batch_runner(
      get_cmds(sim_backtests, runtype=PRERUN),
      cpu=num_cpu, prefix="prepare_config", jobnames=jobnames)
  if flags.FLAGS.run_with_kline_dumper:
    batch_runner(
        get_cmds(sim_backtests, runtype=KLINE),
        cpu=num_cpu, prefix="dump_kline", jobnames=jobnames)
  if flags.FLAGS.fatsim:
    rungroups = collections.defaultdict(list)
    for sim_backtest in sim_backtests:
      tis = tuple((i, tuple(ti)) for i, ti in enumerate(sim_backtest.trading_intervals))
      commonkey = (sim_backtest.config_path, tis)
      rungroups[commonkey].append(sim_backtest)
    fatcmds = []
    jobnamesfat = []
    for (_, trading_intervals), runlist in rungroups.items():
      for subrunlist in chunks(runlist, flags.FLAGS.fatsim):
        for i, trading_interval in trading_intervals:
          jobnamesfat.append(subrunlist[0].gen_jobname(i))
          if getattr(subrunlist[0], "is_midfreq", False):
            fatcmds.append(subrunlist[0].cmd_run_on_date_search(
                subrunlist, trading_interval))
          else:
            fatcmds.append(subrunlist[0].cmd_run_on_date_stratgroups(
                subrunlist, trading_interval, i))
    assert len(fatcmds) == len(jobnamesfat)
    batch_runner(
        fatcmds,
        cpu=num_cpu,
        prefix="runsim",
        jobnames=jobnamesfat)
  else:
    if flags.FLAGS.adjusted_pos_run_hours:
      for sim_backtest in sim_backtests:
        sim_backtest.set_short_term_period()
    if flags.FLAGS.warmup_hours > 0:
      for sim_backtest in sim_backtests:
        sim_backtest.extend_warmup_hours(flags.FLAGS.warmup_by_kline, flags.FLAGS.run_with_kline_dumper)
    sim_cpu_per_job = int(os.environ.get("SIM_CPU_PER_JOB", "1"))
    batch_runner(
        get_cmds(sim_backtests, runtype=RUN, skip_last=use_adjusted_pos),
        cpu=num_cpu, cpu_per_job=sim_cpu_per_job, prefix="runsim", jobnames=jobnames)
  if use_adjusted_pos:
    futures = {}
    with ProcessPoolExecutor(max_workers=6) as executor:
      for i, sim_backtest in enumerate(sim_backtests):
        future = executor.submit(read_ending_pos, sim_backtest)
        futures[future] = i
    for fut in as_completed(futures):
      ret = fut.result()
      if not ret:
        i = futures[fut]
        print(f'fail to set init pos. dir: {sim_backtests[i].tmp_dirname}')
    batch_runner(
        get_cmds(sim_backtests, runtype=RUN),
        cpu=num_cpu, prefix="runsim", jobnames=jobnames)
  if flags.FLAGS.run_with_kline_dumper and flags.FLAGS.remove_tmp_files:
    print("removing cached klines")
    for sim_backtest in sim_backtests:
      if hasattr(sim_backtest, "kline_dir_path"):
        for _, dir_name in sim_backtest.kline_dir_path.items():
          for filename in os.listdir(dir_name):
            if filename.endswith(".pb"):
              os.remove(os.path.join(dir_name, filename))
      if not hasattr(sim_backtest, "kline_cache_dirname"):
        continue
      for filename in os.listdir(sim_backtest.kline_cache_dirname):
        if filename.startswith(f"cached_kline_json.{sim_backtest.kline_prefix}") and filename.endswith(".json"):
          os.remove(os.path.join(sim_backtest.kline_cache_dirname, filename))


def run_batch_for_params(
    sim_type,
    config_files,
    sparams,
    sparams_mdl,
    batch_runner,
    num_cpu,
    tmp_work_root=None):
  if flags.FLAGS.sparams_bm:
    sparams_bm = rsutil.get_sparams(flags.FLAGS.sparams_bm)
  elif flags.FLAGS.run_bm:
    sparams_bm = [{}]
  else:
    sparams_bm = None
  sim_backtests_iter = []
  for config_file in config_files:
    if sparams_bm is not None:
      sim_backtests_iter.extend(get_sim_backtests_for_a_config(
          sim_type,
          config_file,
          sparams=sparams_bm,
          sparam_mdl={},
          tmp_work_root=tmp_work_root))
    for _, sparam_mdl in enumerate(sparams_mdl):
      sim_backtests_raw = get_sim_backtests_for_a_config(
          sim_type,
          config_file,
          sparams,
          sparam_mdl,
          tmp_work_root=tmp_work_root)
      sim_backtests_iter.extend(sim_backtests_raw)
      num_cmds = sum([len(sim_backtest.trading_intervals) for sim_backtest in sim_backtests_iter])
      if flags.FLAGS.fatsim:
        release = num_cmds > flags.FLAGS.fatsim * num_cpu
      else:
        release = num_cmds > 2 * num_cpu
      if release:
        run_sim_backtests(sim_backtests_iter, batch_runner, num_cpu)
        yield sim_backtests_iter
        sim_backtests_iter = []
  if len(sim_backtests_iter) > 0:
    run_sim_backtests(sim_backtests_iter, batch_runner, num_cpu)
    yield sim_backtests_iter
    sim_backtests_iter = []


def remove_tmp_files(sim_backtests):
  removed = []
  for sim_backtest in sim_backtests:
    if os.path.isdir(sim_backtest.tmp_dirname):
      for dirpath, _, filenames in os.walk(sim_backtest.tmp_dirname):
        for filename in filenames:
          filepath = os.path.join(dirpath, filename)
          if os.path.exists(filepath) and os.stat(filepath).st_size > 15000:
            removed.append(filepath)
            try:
              os.remove(filepath)
            except OSError:
              pass


def install_signal_handler():
  print("======================press CTRL+\\ to quit=================")

  def shutdown(signal, frame):
    print("Quitting")
    cleanup_slurm_jobs()
    os._exit(1)

  signal.signal(signal.SIGQUIT, shutdown)


def main(_):
  sparams = rsutil.get_sparams(flags.FLAGS.sparams)
  sparams_mdl = rsutil.get_sparams(flags.FLAGS.sparams_mdl)

  install_signal_handler()

  if flags.FLAGS.pick_first_param:
    sparams = sparams[:1]

  if flags.FLAGS.debug_preset == "plot":
    flags.FLAGS.trading_date = flags.FLAGS.trading_date.split("-")[0]
    flags.FLAGS.hours = flags.FLAGS.hours or "1H"
    flags.FLAGS.split_debug_hours = flags.FLAGS.split_debug_hours or 0.1
    flags.FLAGS.dump_fill_only = 0
    flags.FLAGS.sim_result_dir += "_debug_plot"

  if flags.FLAGS.debug:
    flags.FLAGS.slurm_cpu = 0
    flags.FLAGS.cpu = int(os.environ.get("SIM_CPU_PER_JOB", "1"))
    sparams = sparams[:1]
    sparams_mdl = sparams_mdl[:1]

  if flags.FLAGS.adjusted_pos_run_hours:
    assert flags.FLAGS.use_adjusted_pos

  datetime_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
  tmp_root = os.environ.get('TMPDIR')
  tmp_work_root = os.path.join(tmp_root, f"sim_{datetime_str}")
  os.makedirs(tmp_work_root, exist_ok=True)

  slurm_temp_dir, num_cpu, batch_runner = rsutil.get_batch_driver(
      slurm_cpu=flags.FLAGS.slurm_cpu,
      cpu=flags.FLAGS.cpu,
      tmp_work_root=tmp_work_root)

  if flags.FLAGS.strategy_name is not None:
    assert flags.FLAGS.config_path is None, flags.FLAGS.config_path
    config_files = rsutil.get_config_files(strategy_name=flags.FLAGS.strategy_name)
  else:
    config_files = rsutil.get_config_files(flags.FLAGS.config_path)
    if flags.FLAGS.config_path_exclude is not None:
      config_files_exclude = rsutil.get_config_files(flags.FLAGS.config_path_exclude)
      config_files = list(set(config_files) - set(config_files_exclude))

  if not flags.FLAGS.sim_type:
    guess_sim_type(config_files[0])

  print("sim_result_dir:")
  srd = os.path.realpath(flags.FLAGS.sim_result_dir)
  filesrv = "http://files.ap.ngrok.io"
  srd = f"{filesrv}{srd}".replace(f"{filesrv}//", f"{filesrv}/")
  print(srd)
  rsutil.dump_runsim_command(flags.FLAGS.sim_result_dir)

  print("==STRAT_CONFIG_FILES BEGIN==")
  config_files = list(rsutil.get_config_files_has_symbol(
      config_files, flags.FLAGS.sim_type, flags.FLAGS.pick_symbol_regex))
  print("\n".join(config_files))
  print("==STRAT_CONFIG_FILES   END==")

  print("==PARAMS BEGIN==")
  print(flags.FLAGS.sparams)
  paramdf = pandas.DataFrame(sparams)
  print(tabulate.tabulate(paramdf.head(100).values, headers=paramdf.columns, tablefmt='sql'))
  print("ROW OF PARAMS: ", paramdf.shape[0])
  print("==PARAMS   END==")
  sim_backtests = []
  for sim_backtests_each in run_batch_for_params(
        flags.FLAGS.sim_type,
        config_files,
        sparams,
        sparams_mdl,
        batch_runner,
        num_cpu,
        tmp_work_root):
    if flags.FLAGS.no_analyze:
      continue
    sim_backtests.extend(sim_backtests_each)
    cpu_per_job = 1
    cmds = []
    jobnames = []
    for sim_backtest in sim_backtests_each:
      cmd, cpu = sim_backtest.cmd_analyze_and_plot(flags.FLAGS.compress, flags.FLAGS.analyzer_cpu)
      if isinstance(cmd, list):
        cmds.extend(cmd)
        jobnames.extend([sim_backtest.config_filename] * len(cmd))
      else:
        cmds.append(cmd)
        jobnames.append(sim_backtest.config_filename)
      cpu_per_job = max(cpu, cpu_per_job)
    batch_runner(cmds,
                 cpu=(flags.FLAGS.analyzer_cpu or num_cpu),
                 cpu_per_job=cpu_per_job,
                 prefix="analyze",
                 jobnames=jobnames)
    # wanna flush stats earlier, due to the storage issue.
    if flags.FLAGS.early_remove_tmp_files:
      remove_tmp_files(sim_backtests_each)

  stat_csv_filenames = []
  sparams = []
  for sim_backtest in sim_backtests:
    if isinstance(sim_backtest.stat_csv_filename, list):
      stat_csv_filenames += sim_backtest.stat_csv_filename
      for _ in sim_backtest.stat_csv_filename:
        sparams.append(sim_backtest.sparam)
    else:
      stat_csv_filenames.append(sim_backtest.stat_csv_filename)
      sparams.append(sim_backtest.sparam)

  tmp_dirname = tempfile.mkdtemp(dir=tmp_work_root)
  aggr_spec = {
      'stat_csv_filenames': stat_csv_filenames,
      'sparams': sparams,
  }
  aggr_spec_file = rsutil.create_temp_file_and_dump_json(
      dir=tmp_dirname, prefix="spec.", suffix=".json", data=aggr_spec)
  datetime_str = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f")
  aggr_cmd = "./pyrunner cc/appcoin2/strategy/sim_result_aggr.py" \
      + f" --aggr_spec_file={aggr_spec_file} --sim_result_dir={flags.FLAGS.sim_result_dir}/aggr" \
      + f" --sim_result_postfix={datetime_str}"
  if len(sim_backtests) > 0 and hasattr(sim_backtests[0], "is_focus_mt"):
    aggr_cmd += f" --is_focus_mt={sim_backtests[0].is_focus_mt}"
  batch_runner([aggr_cmd], cpu=1, prefix="aggr")
  if flags.FLAGS.remove_tmp_files:
    remove_tmp_files(sim_backtests)

  if flags.FLAGS.report_email:
    name = 'Sim Backtest Finished'
    content = ' '.join(sys.argv)
    send_mail(name, 'sim_backtest@joomo.io', flags.FLAGS.report_email, name, content)


def define_sim_flags():
  # sparams = search_params
  # --sparams="[dict(a=1,b=2),dict(a=2,b=3)]"
  # --sparams="cartesian(a=[1,2],b=[2,3])"
  flags.DEFINE_string('sparams', None, '')
  flags.DEFINE_string('sparams_mdl', None, '')
  flags.DEFINE_string('sparams_bm', None, '')
  flags.DEFINE_bool('run_bm', False, '')
  flags.DEFINE_integer('fatsim', 0, '')

  # overrides config_path
  flags.DEFINE_string('strategy_name', None, '')
  flags.DEFINE_string('config_path', None, '')
  flags.DEFINE_string('config_path_exclude', None, '')
  # sim_type: rmm
  flags.DEFINE_string('driver_path', None, '')
  flags.DEFINE_string('key_path', None, '')
  # sim_type: lm, basis_smm ?
  flags.DEFINE_string('strat_prefix', None, '')
  flags.DEFINE_string('sim_type', None, '')
  flags.DEFINE_string('trading_date', None, '')
  flags.DEFINE_string('exclude_date', None, '20200802,20200806')
  flags.DEFINE_string('hours', None, '')
  flags.DEFINE_string('hours_warmup', None, '')
  flags.DEFINE_string('pick_symbol_regex', None, '')
  flags.DEFINE_string('sim_result_dir', 'sim_test2', '')
  flags.DEFINE_integer('slurm_cpu', None, None)
  flags.DEFINE_integer('cpu', 20, None)
  flags.DEFINE_integer('analyzer_cpu', 0, None)
  flags.DEFINE_integer('analyzer_mem', 2, "memory per CPU for analyzer in GB")
  flags.DEFINE_bool('recompile', None, '')
  flags.DEFINE_bool('compress', False, '')
  flags.DEFINE_bool('dump_fill_only', False, '')
  flags.DEFINE_string('analyzer_feed_machine', '', '')
  flags.DEFINE_integer('group_dates', None, 'if 3, 3 days sim per once')
  flags.DEFINE_string('split_hours', None, '')
  flags.DEFINE_integer('split_rollover', 0, 'split on friday')
  # debug == 0, no debug mode.
  # debug == 1, debug mode.
  # debug == 2, profile mode.
  flags.DEFINE_integer('debug', 0, '')
  flags.DEFINE_integer('mem_per_cpu_gb', 2, '')
  flags.DEFINE_bool('unfold', False, '')
  flags.DEFINE_bool('use_bbo', str(os.environ.get("use_bbo", None)) == '1', '')
  flags.DEFINE_bool('full_bbo', False, '')
  flags.DEFINE_bool('use_live_latency', False, '')
  flags.DEFINE_bool('use_live_price', False, 'use live price to determin lot_size')
  flags.DEFINE_bool('use_latency_model', False, '')
  flags.DEFINE_bool('exclude_trading_halt', False, 'exclude trading halt time in interval')
  flags.DEFINE_string('report_email', '', '')
  flags.DEFINE_bool('early_remove_tmp_files', False, 'ealry remove tmp files')
  flags.DEFINE_bool('remove_tmp_files', True, 'remove generated files after sim')
  flags.DEFINE_bool('use_adjusted_pos', False,
                    'use last week ending position as init position for next week')
  flags.DEFINE_integer('adjusted_pos_run_hours', 0,
                       'hours that run in first round, 0 for whole period')
  flags.DEFINE_integer('warmup_hours', 0, '')
  flags.DEFINE_boolean('warmup_by_kline', False, '')
  flags.DEFINE_integer('dump_pnl_per_x_minute', 30,
                       'dump pnl per x minute after sim, 0 for not dump')
  flags.DEFINE_bool('bbo_from_cache', True, '')
  flags.DEFINE_bool('include_funding_rate', True, '')
  flags.DEFINE_float('split_debug_hours', None, '')
  flags.DEFINE_string('debug_preset', None, '')
  flags.DEFINE_bool('pick_first_param', False, '')
  flags.DEFINE_bool('no_analyze', False, '')
  flags.DEFINE_string('kline_json', "", "kline info path")
  flags.DEFINE_bool('run_with_kline_dumper', False, "")
  flags.DEFINE_bool('dump_kline_only_once', False, "")
  flags.DEFINE_string('training_date', '', '')
  flags.DEFINE_bool('force_plot_debug_info', False, 'True if you want "each" for multiple trading dates')
  flags.DEFINE_bool('include_mm_stats', False, 'True if you want to include mm stats (Uptime, Order Depth, Volume, ...)')
  flags.DEFINE_bool('include_lp_plot', False, 'Fancy looking black liquidity plot')
  flags.DEFINE_list('price_bp_tholds', [], 'mm_stats price threshold (e.g. 75: midP +/- 75bps)')
  flags.DEFINE_list('quote_tholds', [], 'mm_stats quote threshold (e.g. 100: Check total quote sum of bid/ask orders within price_bp_tholds)')
  flags.DEFINE_list('base_tholds', [], 'mm_stats base threshold (e.g. 100: Check total base qty sum of bid/ask orders within price_bp_tholds)')
  flags.DEFINE_bool('include_option_pnl', False, 'True if you want to include option pnl')
  flags.DEFINE_bool('include_tag_info', False, 'True if you want to include tag info at each txt')
  flags.DEFINE_bool('use_slimfeed', False, 'True if you want to force using slimfeed')
'''
# n file m params slurm
./pyrunner cc/appcoin2/strategy/run_sim.py --sparams \
"cartesian(threshold_bps=[2.0,3.0,4.0])" \
--sim_type=lm \
--config_path "python/coin_deploy/lm_agg2/config_coin1_style_zerothres/*/*.json" \
--trading_date=20200309-20200325 --hours=24H --slurm_cpu=150 \
--pick_symbol_regex=.*QUARTER

# n file m params search
./pyrunner cc/appcoin2/strategy/run_sim.py --sparams \
"cartesian(threshold_bps=[2.0,3.0,4.0])" \
--sim_type=lm \
--config_path "python/coin_deploy/lm_agg2/config_coin1_style_zerothres/*/*.json" \
--trading_date=20200309-20200325 --hours=12H --cpu=50 \
--pick_symbol_regex=.*QUARTER

# 1 file m params search
./pyrunner cc/appcoin2/strategy/run_sim.py \
--sparams \
"cartesian(threshold_bps=[2.5,3.5],lean_per_stack_bps=[0,1])" \
--sim_type=lm --config_path \
python/coin_deploy/lm_agg2/lm_agg_huobi_futures_allexpiry_btc.json \
--trading_date=20200324 --hours=1H
'''
if __name__ == '__main__':
  define_sim_flags()
  logging.basicConfig(level='DEBUG', format='%(levelname)8s %(asctime)s %(name)s] %(message)s')
  app.run(main)
